﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using AngleSharp;
using AngleSharp.Dom;
using AngleSharp.Extensions;
using AngleSharp.Parser.Html;
using Common.Dao;
using Common.Extension;
using Common.Model;
using log4net;
using Newtonsoft.Json;
using Spider.Service;

namespace Spider.Spider
{
    public class YunFincalSpider
    {
        private static readonly ILog Log = LogManager.GetLogger(typeof(YunFincalSpider));

        public static readonly string From = "云财经";
        public static readonly string ID_ = "ycj_";

        public static readonly string SiteBaseUrl = @"http://www.yuncaijing.com/insider/page_";

        public static readonly string Html = @".html";

        public static readonly string SectionClassName = @"main fl";
        public static readonly string TimeClassName = @"time";
        public static readonly string TitleClassName = @"title";
        public static readonly string TypeClassName = @"type";
        public static readonly string Href = @"href";
        public static readonly string AbsClassName = @"des";

        public static readonly string Path = @".\cookie";

        public static readonly string TesterKey = "yunfincal";
        
        public YunFincalSpider()
        {
            var word = YunFinFilterWordDao.GetWord();
            TitleFilterWords.AddRange(word);
            TitleFilterWords.AddRange(TitleFilterWord);
            var tester = YunFincalSpider.GetTester();
            IpProxyPool.AddTester(YunFincalSpider.TesterKey,tester);
        }

        public HashSet<string> TitleFilterWords = new Quartz.Collection.HashSet<string>();

        public static readonly string[] TitleFilterWord =
        {
            "逆回购", "期货", "早间公告", "晚间公告", "午间公告", "净流入", "净流出", "北上资金", "加仓", "建仓", "中小创", "个股",
            "减仓", "机构", "监管", "证监会", "银监会", "股指", "深交所", "深圳交易所", "上海交易所", "上交所",
            "港交所", "跌停", "涨停", "问询函", "交易所", "上海清算所", "证监会", "央行", "亿元", "净投放", "涨停分析", "飙升", "暴跌",
            "跌停分析", "人民币", "美元", "离岸", "外汇", "港资", "港股通", "深股通", "沪股通", "收盘点评", "复盘", "冲高", "爆拉", "急拉",
            "冲高回落", "早盘", "午盘", "尾盘", "在岸", "公募", "私募", "基金", "国家队", "净利润", "开盘", "收盘", "大涨", "大跌", "超跌",
            "净回笼", "新股", "申购", "赎回", "债券", "国债", "可转债", "要闻", "精华", "头条", "头版", "报纸", "夜盘", "急跌", "小幅", "大幅",
            "外围", "VIP", "云财经", "人民币兑", "美元兑", "汇率", "交易所", "中签", "债市", "st", "ST", "*ST", "*st", "打折", "促销", "大促",
            "货币政策", "报会", "两市", "沪深", "指数", "融资", "余额", "财政", "货币总量", "总量", "商品", "货币", "板块", "最后",
            "走强", "减弱", "走弱", "临停", "停牌", "临时停牌", "复牌", "分红", "送股", "高送转", "配股", "审核", "询问", "发审委",
            "午后", "回调", "跳水", "翻红", "翻绿", "涨停", "跌停", "午间", "点评", "异动", "解禁", "定增", "配资", "回落",
            "恒指", "恒生指数", "纳指", "纳斯达克", "道指", "道琼斯", "美股", "港股", "沪指", "上证", "深指", "三大指数", "股价",
            "创业板", "中小板", "拉升", "领涨", "领跌", "提示：", "风险", "打折", "云财经", "质押", "股票", "退市", "新闻联播",
            "总汇", "汇总", "年报", "预增", "波段", "收割者", "利率", "利率债", "投资者", "投资者数", "中长线", "白马股",
            "优惠", "打折", "最后一天", "开盘", "蓝筹", "分级B", "上折", "下折", "etf", "ETF", "LOF", "lof", "场内", "场外",
            "PMI", "上期所", "深期所", "交易日", "CPI", "PPI", "休市", "A/H", "A股", "B股", "收涨", "涨近", "收跌", "跌近", "走高", "GDP",
            "短讯", "快讯"
        };

        public static readonly string[] TypeFilterWord =
        {
            "盘中快报", "两融余额", "证监会例会", "年报", "个股利空", "个股利好"
        };

        public List<News> GetNews(DateTime startTime, DateTime endTime)
        {
            var config = Configuration.Default.WithDefaultLoader();
            config.WithCookies();
            config.WithCss();
            config.WithJavaScript();

            var address = SiteBaseUrl;

            var contentList = new List<News>();
            int i = 0;
            while (true)
            {
                i++;
                var url = address + i + Html;

                var html = GetHtml(url);
                var parser = new HtmlParser();
                var document = parser.Parse(html);
                var cellSelector = "section";
                var sections = document.QuerySelectorAll(cellSelector);
                var mainFl = sections.Where(m => SectionClassName.Equals(m.ClassName));
                var section = mainFl.First();
                var ul = section.QuerySelectorAll("ul").First(m => "news-ul".Equals(m.GetAttribute("class")));
                var querySelectorAll = ul.QuerySelectorAll("li");
                List<News> contents = querySelectorAll.Select(SelectContentFromLi).ToList();

                contents = contents.Where(m =>
                {
                    //过滤个股资讯
                    string title = m.Title.Replace(" ", "");
                    bool filter = StockCheckerService.ContainStock(title)
                                  || TypeFilterWord.Any(w => m.Type.Contains(w))
                                  || TitleFilterWords.Any(w => m.Title.Contains(w))
//                                  || TitleFilterWord.Any(w => m.Abs.Contains(w))
                                  && !(m.Title.Contains("央行") && m.Title.Contains("数字货币"))
                                  && !(m.Title.Contains("央行") && m.Title.Contains("虚拟货币"))
                                  && !(m.Title.Contains("央行") && m.Title.Contains("电子货币"))
                                  && !(m.Title.Contains("央行") && m.Title.Contains("区块链"))
                                  && !(m.Title.Contains("上交所") && m.Title.Contains("区块链"))
                                  && !(m.Title.Contains("上交所") && m.Title.Contains("数字货币"))
                                  && !(m.Title.Contains("上交所") && m.Title.Contains("虚拟货币"))
                                  && !(m.Title.Contains("上交所") && m.Title.Contains("电子货币"))
                                  && !(m.Title.Contains("监管") && m.Title.Contains("区块链"))
                                  && !(m.Title.Contains("监管") && m.Title.Contains("数字货币"))
                                  && !(m.Title.Contains("监管") && m.Title.Contains("虚拟货币"))
                                  && !(m.Title.Contains("监管") && m.Title.Contains("电子货币"))
                                  && !(m.Title.Contains("场内") && m.Title.Contains("电子货币"))
                                  && !(m.Title.Contains("场内") && m.Title.Contains("区块链"))
                                  && !(m.Title.Contains("场内") && m.Title.Contains("数字货币"))
                                  && !(m.Title.Contains("场内") && m.Title.Contains("虚拟货币"))
                                  && !(m.Title.Contains("场外") && m.Title.Contains("区块链"))
                                  && !(m.Title.Contains("场外") && m.Title.Contains("数字货币"))
                                  && !(m.Title.Contains("场外") && m.Title.Contains("虚拟货币"))
                                  && !(m.Title.Contains("场外") && m.Title.Contains("电子货币"))
                                  && !(m.Title.Contains("交易所") && m.Title.Contains("区块链"))
                                  && !(m.Title.Contains("交易所") && m.Title.Contains("数字货币"))
                                  && !(m.Title.Contains("交易所") && m.Title.Contains("虚拟货币"))
                                  && !(m.Title.Contains("交易所") && m.Title.Contains("电子货币"))
                        ;
                    return !filter;
                }).ToList();

                if (contents.Any(content => content.SiteTime <= startTime))
                {
                    contentList.AddRange(contents.Where(content => content.SiteTime > startTime));
                    break;
                }
                contentList.AddRange(contents);
            }

            return
                contentList.Where(m => m.SiteTime < endTime)
                    .Where(m => !(m.Abs?.Contains("云财经") ?? false) && !(m.Content?.Contains("云财经") ?? false))
                    .ToList();
        }


        private static void GetStocks(IConfiguration config, List<News> news)
        {
            var tasks = news.Select(m => BrowsingContext.New(config).OpenAsync(m.ContentLink)).ToList();
            Task.WaitAll(tasks.ToArray());

            for (int i = 0; i < news.Count; i++)
            {
                var aList = tasks[i].Result.QuerySelectorAll("a");
                var stocks = aList.Where(m => null != m.GetAttribute("data-showchart-code"))
                    .Select(m => m.GetAttribute("data-showchart-code")).ToList();
//                Console.WriteLine(tasks[i].Result.ToHtml());
                if (stocks.Any())
                {
                    var stringBuilder = new StringBuilder();
                    foreach (var stock in stocks)
                    {
                        stringBuilder.Append(stock);
                        stringBuilder.Append(",");
                    }
                    stringBuilder.Remove(stringBuilder.Length - 1, 1);
                    news[i].Stocks = stringBuilder.ToString();
                }

//                var result = tasks[i].Result.QuerySelectorAll("section")
//                    .Where(e => "news-mate".Equals(e.ClassName))
//                    .Select(m => m).ToList();

//                if (result.Any())
//                {
//                    var stocks =  result.First().QuerySelectorAll("a")
//                        .Where(m => null != m.GetAttribute("data-showchart-code"))
//                        .Select(m => m.GetAttribute("data-showchart-code"));
//
//                    var stringBuilder = new StringBuilder();
//                    foreach (var stock in stocks)
//                    {
//                        stringBuilder.Append(stock);
//                        stringBuilder.Append(",");
//                    }
//                    stringBuilder.Remove(stringBuilder.Length - 1, 1);
//                    news[i].Stocks = stringBuilder.ToString();
//                }
            }
        }

        private static News SelectContentFromLi(IElement element)
        {
            var time = element.QuerySelector(TimeClassName).TextContent;
            var titleDiv = element.QuerySelectorAll("a").Where(e => TitleClassName.Equals(e.ClassName)).ToList();
            var title = titleDiv.Select(e => e.GetAttribute(TitleClassName)).First();
            var href = titleDiv.Select(e => e.GetAttribute(Href)).First();
            var content = element.QuerySelectorAll("span").Where(e => AbsClassName.Equals(e.ClassName))
                .Select(m => m.TextContent).First();
            var id = element.GetAttribute("data-id");
            var typeDiv = element.QuerySelectorAll("a").Where(e => TypeClassName.Equals(e.ClassName)).ToList();

            var type = "";
            if (typeDiv.Count > 0)
            {
                type = typeDiv.Select(e => e.Text()).First();
            }
            time = time.Replace("今天 ", "");
            char[] sp = {' '};
            var strings = time.Split(sp, StringSplitOptions.RemoveEmptyEntries);
            if (strings.Length == 2)
            {
                time = strings[1] + " " + strings[0];
            }
            return new News
            {
                Type = type,
                Title = title.Replace("云财经讯，", "").Replace("预告：", "").Replace("媒体：", "").Replace("云财经App讯，", "")
                    .Replace("云财经APP讯,", ""),
                SiteTime = DateTime.Parse(time),
                ContentLink = SiteBaseUrl + href,
                Abs = content.Replace("云财经讯，", "").Replace("云财经App讯，", "").Replace("云财经APP讯,", ""),
                From = From,
                Tag = "",
                ContentId = ID_ + id
            };
        }

        private static HttpWebResponse GetResponse(string url)
        {
//            HttpWebRequest requestMain = (HttpWebRequest)WebRequest.Create("http://www.yuncaijing.com/");
//            requestMain.Method = WebRequestMethods.Http.Get;
//            requestMain.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
//            requestMain.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
//            requestMain.Headers.Add("Origin:http://www.yuncaijing.com");
//            requestMain.Headers.Add("Pragma:no-cache");
//            requestMain.Headers.Add("X-Requested-With:XMLHttpRequest");
//            requestMain.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36";
//            HttpWebResponse webResponse = (HttpWebResponse)requestMain.GetResponse();
//
//            string cookie = webResponse.Headers.Get("Set-Cookie");
//            string html = getHtml(GetCookieName(cookie), GetCookieValue(cookie));

//            var proxyInfo = IpProxyPool.GetIpProxy(TesterKey);
//            var webProxy = new WebProxy(proxyInfo.Ip,proxyInfo.Port);
        
            HttpWebRequest request = (HttpWebRequest) WebRequest.Create(url);

            request.Method = WebRequestMethods.Http.Get;

            request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
            request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
            request.Headers.Add("Origin:https://www.yuncaijing.com/waf_auth/check_code_page.html");
            request.Headers.Add("Pragma:no-cache");
            request.Referer = "Referer:https://www.yuncaijing.com/waf_auth/check_code_page.html";
            //            request.Headers.Add("Referer:" + url);
            request.Headers.Add("X-Requested-With:XMLHttpRequest");
            request.UserAgent =
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36";
//            request.Proxy = webProxy;
            SetCookie(request);

            HttpWebResponse response;
            try
            {
                response = (HttpWebResponse)request.GetResponse();
            }
            catch (Exception e)
            {
//                IpProxyPool.RemoveProxy(TesterKey,proxyInfo);
                Log.Error(e);
                return GetResponse(url);
            }

            return response;
        }

        private static void SetCookie(HttpWebRequest request)
        {
            var dictionary = new Dictionary<string, string>();
            var exists = File.Exists(Path);
            if (!exists)
            {
//                request.Headers.Add("Cookie", "PHPSESSID=cvf3b0r4o69615g1gtp6r57631");
//                request.Headers.Add("Cookie", "YUNSESSID=r97chkdsmb523hff95lrdnroh5");
//                request.Headers.Add("Cookie", "ycj_wafsid=wafsid_8514c92f522a7e4d763e823204623ecb");
                request.Headers.Add("Cookie",
                    "YUNSESSID=71kbhbub13oec7b2mmskfivmj5; __root_domain_v=.yuncaijing.com; _qddaz=QD.6iriv0.rlrfmc.jk7ntn7w; CNZZDATA1257026985=1619301752-1528159677-null%7C1535093709; wx_state=g5zTQbEBPKtxy4ym; _qddab=3-db61ir.jnfdjpu6; ycj_wafsid=wafsid_9219de28afcdbe7823c438ed0d457b71; PHPSESSID=mtrfgm9277a229jaidsl4mmvb4; ycj_uuid=84515c470fbfc98d45a6eaaefc13c1b4; ycj_locate=aHR0cHM6Ly93d3cueXVuY2FpamluZy5jb20v; Qs_lvt_168612=1542329763%2C1543194075%2C1543278575%2C1544750884%2C1548117203; Hm_lvt_b68ec780c488edc31b70f5dadf4e94f8=1548117208; Qs_pv_168612=1509446623180799200%2C4216900890427465000%2C4505343800754019300%2C2321999816106521000%2C4396156300392361500; ycj_from_url=aHR0cHM6Ly93d3cueXVuY2FpamluZy5jb20v; Hm_lpvt_b68ec780c488edc31b70f5dadf4e94f8=1548117216; ycj_main_token=ycj_main_2c02b040b65b4c34815a62c7625e201b_c61a805e43e21bf90af16eb124d1607f");
//                dictionary.Add("PHPSESSID","cvf3b0r4o69615g1gtp6r57631");
//                dictionary.Add("YUNSESSID","r97chkdsmb523hff95lrdnroh5");
//                dictionary.Add("ycj_wafsid","wafsid_8514c92f522a7e4d763e823204623ecb");
                dictionary.Add("ycj_uuid", "84515c470fbfc98d45a6eaaefc13c1b4");
                var serializeObject = JsonConvert.SerializeObject(dictionary);
                File.WriteAllText(Path, serializeObject);
            }
            else
            {
                string json = File.ReadAllText(Path);
                var dic = JsonConvert.DeserializeObject<Dictionary<string, string>>(json);
                dictionary = dic;
                foreach (var keyValuePair in dic)
                {
                    request.Headers.Add("Cookie", keyValuePair.Key + "=" + keyValuePair.Value);
                }
            }
        }

        /// <summary>
        /// </summary>
        private static string GetHtml(string url)
        {
            var response = GetResponse(url);
       
        
            var cookiesString = response.Headers.Get("Set-Cookie");
            var str1 = new char[] {';', ','};
            var str2 = new char[] {'='};
            var coookies = cookiesString.Split(str1, StringSplitOptions.RemoveEmptyEntries).Where(
                m => m.Contains(
                    "ycj_uuid") /*||m.Contains("ycj_from_url")||m.Contains("ycj_locate")||m.Contains("YUNSESSID")*/);

            string json = File.ReadAllText(Path);
            var dic = new Dictionary<string, string>();

            foreach (string cookie in coookies)
            {
                string[] strings = cookie.Split(str2, StringSplitOptions.RemoveEmptyEntries);
                if (dic.ContainsKey(strings[0]))
                {
                    dic.Remove(strings[0]);
                }
                dic.Add(strings[0], strings[1]);
            }

            var serializeObject = JsonConvert.SerializeObject(dic);
            File.WriteAllText(Path, serializeObject);
            Stream myResponseStream = response.GetResponseStream();
            StreamReader myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8"));
            string retString = "";
            try
            {
                myResponseStream = response.GetResponseStream();
                myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8"));
                retString = myStreamReader.ReadToEnd();
            }
            catch (Exception e)
            {
                Log.Error(e);
                throw;
            }
            finally
            {
                myStreamReader?.Close();
                myResponseStream?.Close();
                response?.Close();
            }

            return retString;
        }

        public static IpProxyPool.Tester GetTester()
        {
            string url = "https://www.yuncaijing.com";
            return (ip, port) =>
            {
                HttpWebRequest request = (HttpWebRequest) WebRequest.Create(url);

                request.Method = WebRequestMethods.Http.Get;

                request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
                request.Accept =
                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
//                request.Headers.Add("Origin:https://www.yuncaijing.com/waf_auth/check_code_page.html");
                request.Headers.Add("Pragma:no-cache");
//                request.Referer = "Referer:https://www.yuncaijing.com/";
                request.Headers.Add("X-Requested-With:XMLHttpRequest");
                request.UserAgent =
                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36";

                SetCookie(request);

                WebResponse response = null;

                Stream myResponseStream = null;
                StreamReader myStreamReader = null;
                string html = "";
                try
                {
                    Task<WebResponse> responseAsync = request.GetResponseAsync();
                    responseAsync.Wait(500);
                    if (responseAsync.Status == TaskStatus.RanToCompletion)
                    {
                        response = responseAsync.Result;
                        myResponseStream = response.GetResponseStream();
                        myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8"));
                        html = myStreamReader.ReadToEnd();
                    }
                }
                catch (Exception e)
                {
                    Log.Error(e);
                }
                finally
                {
                    myStreamReader?.Close();
                    myResponseStream?.Close();
                    response?.Close();
                }

                return html.Contains("首页");
            };
        }
    }
}